<?php
include_once "./abbreviations.php";
include_once "./SmileLaughDetenction.php";

class Normalizer {
	
	private $dict;
	private $abbreviations;
	private $smileLaughDetector;
	
	public function __construct() {}
	
	private function train() {
			$this -> dict = json_decode(file_get_contents("./src/dictionary.txt"), true);
/*			$contents = file_get_contents($file);
			echo "end read dic at ".date("D M j G:i:s T Y")."<br />";
			// get all strings of word letters			
			$tmp = split('#',$contents);
			
			for($i = 0 ; $i < (count($tmp)/2) ; $i = $i+2)
			{
				$dictionary[$tmp[$i]] = $tmp[$i+1];
			}
		    preg_match_all('/#/', $contents, $matches[$con]);
			unset($contents);
			$dictionary = array();
			foreach($matches[0] as $word) {
				$word = strtolower($word);
				if(!isset($dictionary[$word])) {
						$dictionary[$word] = 0;
				}
				$dictionary[$word] += 1;
			}
			unset($matches);
			print_r(json_encode($dictionary));
			return $dictionary;
*/	}
	
	
	private function correct($word) {
			$dictionary = $this -> dict;
			$word = strtolower($word);
			if(isset($dictionary[$word])) {
					return $word;
			}
	
			$edits1 = $edits2 = array();
			foreach($dictionary as $dictWord => $count) {
				$dist = levenshtein($word, $dictWord);
				if($dist == 1) {
						$edits1[$dictWord] = $count;
				} else if($dist == 2) {
						$edits2[$dictWord] = $count;
				}
			}
	
			if(count($edits1)) {
					arsort($edits1);
					return key($edits1);
			} else if(count($edits2)) {
					arsort($edits2);
					return key($edits2);
			}
			// Nothing better
			return $word;
	}
	
	private function correct1($word) {
			$dictionary = $this -> dict;
			$word = strtolower($word);
			if(isset($dictionary[$word])) {
					return $word;
			}
	
			$edits1 = $edits2 = array();
			foreach($dictionary as $dictWord => $count) {
				$dist = levenshtein($word, $dictWord);
				if($dist == 1) {
						$edits1[$dictWord] = $count;
				} else if($dist == 2) {
						$edits2[$dictWord] = $count;
				} else if($dist == 3) {
						$edits3[$dictWord] = $count;
				} else if($dist == 4) {
						$edits4[$dictWord] = $count;
				} else if($dist == 5) {
						$edits5[$dictWord] = $count;
				}
			}
	
			if(count($edits1)) {
					arsort($edits1);
					return key($edits1);
			} else if(count($edits2)) {
					arsort($edits2);
					return key($edits2);
			} else if(count($edits3)) {
					arsort($edits3);
					return key($edits3);
			} else if(count($edits4)) {
					arsort($edits4);
					return key($edits4);
			} else if(count($edits5)) {
					arsort($edits5);
					return key($edits5);
			}
	
			// Nothing better
			return $word;
	}
	
	/* Function that splits a text into different sentences */
	private function textToSentence($text){
	
		$text = ereg_replace("(\t)+","",$text);           
		$text = preg_replace("/\n\s*\n+/", "\n\n", $text);
	//	$text = preg_replace("/[\n]{3,}/", "\n\n", $text);
	 
		$sentences = array();
		$a = split("\n\n",$text);   
		foreach ($a as $b)
		{
			$b = preg_replace("/http:\/\/(.*?)[\s\)]/", "", $b);
			$b = preg_replace("/http:\/\/([^\s]*?)$/", "", $b);
			$b = preg_replace("/\[\s*[0-9]*\s*\]/", "", $b);
			foreach (split('[(.?|(\(.?*\))|\!)]',$b) as $sent)
				if(strlen(trim($sent)) > 0){
					$sent = preg_replace("/\n/", " ", $sent);
					$sent = trim(ereg_replace("  ", " ", $sent));
					$sent[0] = strtoupper($sent[0]);
					array_push($sentences,$sent);
				}
		}
		return $sentences;
	}
		
	private function extractWordsFromSentence($sentence)
	{    
		$sentence = strtolower($sentence);
	
		preg_match_all('/\w+/i', $sentence, $matches);
		
		for ($i = 0; $i < count($matches[0]); $i++)
			$matches[0][$i] = str_replace('_', '', $matches[0][$i]);
	
		return $matches[0];
	/*
		// ponctuation caracters
		$toreplace = array(",",".",";",":","!","?","(",")","[","]","{","}",'"',"^",'$','#','%','_', '-');
		// We replace them by a space
		$replacedBy = " "; 
		
		// We apply the changes
		$withoutPonctuation = trim(str_replace($toreplace, $replacedBy, $sentence)); 
		// We put every words in an array
		$separator = "[ ]+"; // 1 or more space
		$words = split($separator, $withoutPonctuation); 
		return $words;
		*/
	}
	
	/* Function to normalize the words of a given text */
	public function normalize($message)
	{
		if (!$this -> dict)
			$this -> train("./src/dictionary.txt");
		if (!$this -> abbreviations)
			$this -> initializeAbbreviations();
		$this -> smileLaughDetector = new SmileLaughDetector();
				
		// split text into an array of sentences
		$sentences = $this -> textToSentence($message);
		
		$i = 0;
		foreach($sentences as $sentence)
		{
			// split sentences into an array of words
			$words = $this -> extractWordsFromSentence($sentence);
			
			// At this point we have an array of word for each sentence. Proceed normalizing them!
			$normalizedSentence = array();
	
			for ($w = 0; $w < count($words); $w++)
			{
				$word = $words[$w]; 
	//		foreach ($words as $word) {
				// if the current word is an abbreviation, put in the sentence the new words.
				if ($this -> abbreviations[$word]) {
	//				echo "FOUND ABBREVIATION: ".print_r(explode(' ', $abbreviations[$word]), true);
					$words_tmp = array();
					foreach (array_slice($words, 0, $w) as $curr)
						array_push($words_tmp, $curr);	
					foreach (explode(' ', $this -> abbreviations[$word]) as $curr)
						array_push($words_tmp, $curr);	
					foreach (array_slice($words, $w + 1) as $curr)
						array_push($words_tmp, $curr);
					$words = $words_tmp;
					$word = $words[$w];
				}
	
				// check whether a word is a laughing
				if (!$this -> smileLaughDetector -> isLaughing($word)) {
					// remove repetitions of letters (e.g. 'goooddd' becomes 'good') 
					$word_tmp = $this -> removeRepetitions ($word);
					// if in the previous step the word has been modified, it can be mispelled --> correct it!!
					if ($word_tmp != $word)
						$correctedWord = $this -> correct1($word_tmp);
					else
						$correctedWord = $this -> correct($word);
	
					// put in the new sentence the normalized word 
					array_push($normalizedSentence, $correctedWord);
				}
			}
			$i++;
			$normalizedPost[$i] = $normalizedSentence;
		}
		return $normalizedPost;
	}
	
	
	/* Function that removes multiple occurrences of a same letter in a word (e.g. "woooooooooowwww" becomes "woow") */
	private function removeRepetitions ($word) {
		
		$word = strtolower($word);
		
		for ($i = 2; $i < strlen($word); $i++) {
			$prec2 = $word[$i - 2];
			$prec1 = $word[$i - 1];
			$corr = $word[$i];
	
			if ($prec2 == $prec1 && $prec1 == $corr) 
			{
				$j = $i;
	
				while ($j < strlen($word) && $corr == $word[$j])
					$j++;
	
				$tmp2 = substr($word, $j - 1, strlen($word));
				$tmp1 = substr($word, 0, $i - 1);
				$word = $tmp1.$tmp2;
			}
		}
		
		$vowels = array('a' => 1, 'e' => 1, 'i' => 1, 'o' => 1, 'u' => 1, 'g' => 1);
		if (!$vowels[$word[0]] && $word[0] == $word[1])
			$word = substr($word, 1, strlen($word));
		
		return $word;
	}
	
	private function initializeAbbreviations() {
		$this -> abbreviations = array(
		"10x" => "thanks",
		"10q" => "thank you",
		"2" => "too",
		"2h" => "two hands",
		"3ad" => "thread",
		"4" => "for",
		"3d" => "thread",
		"afaik" => "as far as i know",
		"afair" => "as far as i remember",
		"aiui" => "as I understood it",
		"aka" => "also known as",
		"afk" => "away from keyboard",
		"asap" => "as soon as possible",
		"asl" => "age sex location",
		"b4" => "before",
		"bamf" => "Bad Ass Mother Fucker",
		"bb" => "bye bye",
		"bbl" => "be back later",
		"bbq" => "barbecue",
		"brb" => "be right back",
		"btw" => "by the way",
		"biz" => "business",
		"bff" => "best friends forever",
		"cause" => "because",
		"cos" => "because",
		"cuz" => "because",
		"cu" => "see you",
		"cul8r" => "see you later",
		"cy" => "see you",
		"cya" => "see you around",
		"cw" => "caln war",
		"dtfm" => "death to false metal",
		"dnd" => "do not disturb",
		"dps" => "damage per second",
		"dunno" => "don't know",
		"esad" => "eat shit and die",
		"f2p" => "free to play",
		"faq" => "frequently asked questions",
		"ff" => "fuck off",
		"ffs" => "for fuck sake",
		"fg" => "fucking good",
		"fml" => "fuck my life",
		"fotl" => "falling off the toilet laughing",
		"ftv" => "for the victory",
		"ftw" => "engrish for the win",
		"fyi" => "for your information",
		"fu" => "fuck you",
		"fy" => "fuck you",
		"gj" => "good job",
		"gg" => "good game",
		"gz" => "congratulations",
		"giyf" => "google is your friend",
		"gl" => "good luck",
		"gratz" => "congratulations",
		"gr8" => "great",
		"g2g" => "go to go",
		"gtfo" => "get the fuck out",
		"gtg" => "go to go",
		"gn" => "good night",
		"gga" => "good game all",
		"h8" => "hate",
		"hbu" => "how about you",
		"hf" => "have fun",
		"hth" => "hope this help",
		"hya" => "hello",
		"idc" => "I don't care",
		"idk" => "I don't know",
		"idts" => "I don't think so",
		"iirc" => "if I remember correctly",
		"imao" => "in my arrogant opinion",
		"imho" => "in my humble opinion",
		"irl" => "real life",
		"ily" => "I love you",
		"ilysm" => "I love you so much",
		"imo" => "In my opinion",
		"imnsho" => "In My Not So Humble Opinion",
		"jk" => "just kidding",
		"Jam" => "just a minute",
		"k" => "ok",
		"kthxbye" => "oh thanks bye",
		"l2p" => "learn to play",
		"l2l" => "learn to lose",
		"l8r" => "later",
		"lfm" => "looking for more",
		"lmao" => "laughing my ass off",
		"lmfao" => "laughing my fuck ass off",
		"lol" => "laughing out loud",
		"m8" => "mate",
		"mof" => "male or female",
		"msg" => "message",
		"mwah" => "kiss",
		"mwa" => "kiss",
		"nm" => "not much",
		"nn" => "nice night",
		"n8" => "night",
		"nope" => "no",
		"np" => "no problem",
		"nvm" => "nevermind",
		"ntmu" => "nice to meet you",
		"ntmu2" => "nice to meet you too",
		"omg" => "oh my god",
		"omfg" => "oh my fucking god",
		"omw" => "on my way",
		"o" => "oh",
		"rly" => "oh really",
		"obv" => "obviously",
		"ofc" => "of course",
		"plz" => "please",
		"pls" => "please",
		"ppl" => "people",
		"prolly" => "probably",
		"rfc" => "request of comments",
		"rl" => "real life",
		"r" => "are",
		"rotfl" => "rolling on the floor laughing",
		"rotflastc" => "rolling on the floor laughing and scaring the cat",
		"rotflmao" => "rolling on the floor laughing my ass off",
		"rtfm" => "read the fine manual",
		"rehi" => "hello again",
		"ruok" => "are you ok",
		"smack" => "kiss",
		"stfu" => "shut the fuck up",
		"sup" => "what's up",
		"soz" => "sorry",
		"snafu" => "Situation Normal All Fucked Up",
		"sry" => "sorry",
		"tc" => "take care",
		"tia" => "thanks in advance",
		"tlnr" => "too long not read",
		"ttyl" => "talk to you later",
		"ty" => "thanks",
		"thx" => "thanks",
		"tnx" => "thanks",
		"u" => "you",
		"ur" => "you're",
		"ul" => "unluckly",
		"ust" => "I used to",
		"uw" => "you're welcome",
		"um" => "your mum",
		"wanna" => "want to",
		"wataya" => "what do you",
		"w8" => "wait",
		"wb" => "welcome back",
		"wbu" => "what about you",
		"wd" => "well done",
		"wombat" => "waste of money brain and time",
		"wp" => "well played",
		"wtf" => "what the fuck",
		"wth" => "what the hell",
		"wtb" => "want to buy",
		"wts" => "want to sell",
		"wtt" => "want to trad",
		"wubu2" => "what have you been up to",
		"wtk" => "what the keyboard",
		"Y" => "yes",
		"ya" => "you",
		"yr" => "your",
		"yw" => "you are welcome");
	}
}
?>